const axios = require('axios');
const axiosRetry = require('axios-retry').default;
const { isNetworkError, isRetryableError } = require('axios-retry');
const cheerio = require('cheerio');
const fs = require('fs');
const path = require('path');
const schedule = require('node-schedule');

// 创建数据存储目录
const dataDir = path.join(__dirname, '../data');
if (!fs.existsSync(dataDir)) {
  fs.mkdirSync(dataDir, { recursive: true });
}

// 添加请求超时和重试机制
// 配置axios实例
const api = axios.create({
  timeout: 20000 // 增加超时时间到20秒
});

// 添加随机User-Agent池
const userAgents = [
  // Chrome on Windows
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
  // Safari on macOS
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Safari/605.1.15',
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
  // Firefox
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0',
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4; rv:109.0) Gecko/20100101 Firefox/115.0',
  // Edge
  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.67',
  // Chrome on Linux
  'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',
  // Mobile User-Agents
  'Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1',
  'Mozilla/5.0 (Android 13; Mobile; rv:109.0) Gecko/115.0 Firefox/115.0'
];

// 请求拦截器 - 每次请求随机选择User-Agent
// 请求拦截器 - 添加浏览器特征请求头
api.interceptors.request.use(config => {
  const randomIndex = Math.floor(Math.random() * userAgents.length);
  config.headers = {
    'User-Agent': userAgents[randomIndex],
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'en-US,en;q=0.5',
    'Referer': 'https://github.com/',
    'DNT': '1',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
    'Sec-Fetch-Dest': 'document',
    'Sec-Fetch-Mode': 'navigate',
    'Sec-Fetch-Site': 'same-origin',
    'Sec-Fetch-User': '?1',
    'Cache-Control': 'no-cache',
    'Pragma': 'no-cache'
  };
  return config;
});

// 配置重试策略
axiosRetry(api, {
  retries: 7, // 增加重试次数到7次
  retryDelay: (retryCount) => {
    // 增加基础延迟并添加随机抖动
    const baseDelay = retryCount ** 2 * 1500; // 1.5s, 6s, 13.5s, 24s, 37.5s, 54s, 73.5s
    const jitter = Math.random() * 2000; // 0-2秒随机抖动
    return baseDelay + jitter;
  },
  retryCondition: (error) => {
    // 网络错误、5xx状态码或429 Too Many Requests时重试
    return isNetworkError(error) || isRetryableError(error) || (error.response && error.response.status === 429);
  }
});

// 爬取Trending Repositories
async function crawlRepositories(since = 'daily') {
  const url = since === 'daily' ? 'https://github.com/trending' : `https://github.com/trending?since=${since}`;
  try {
    const response = await api.get(url);
    const $ = cheerio.load(response.data);
    const repositories = [];

    $('.Box-row').each((index, element) => {
      const repo = $(element);
      const title = repo.find('h2 a').text().trim();
      const [owner, repoName] = title.split('/').map(part => part.trim());
      // 提取总stars数量
      const starsText = repo.find('.f6 a.Link--muted').eq(0).text().trim();
      const stars = parseInt(starsText.replace(/[^\d,]/g, '').replace(/,/g, ''), 10) || 0;

      // 提取stars增长数据
      const starsGrowthText = repo.find('.f6 .float-sm-right').text().trim() || '';
      const starsGrowth = parseInt(starsGrowthText.replace(/[^\d]/g, ''), 10) || 0;

      // 提取forks数量
      const forksText = repo.find('.f6 a.Link--muted').eq(1).text().trim();
      const forks = parseInt(forksText.replace(/[^\d,]/g, '').replace(/,/g, ''), 10) || 0;
      const language = repo.find('[itemprop="programmingLanguage"]').text().trim() || '';
      const description = repo.find('p').text().trim() || '';
      const url = `https://github.com${repo.find('h2 a').attr('href') || ''}`;

      repositories.push({
        name: repoName,
        owner,
        url,
        description,
        language,
        stars,
        starsGrowth,
        starsGrowthText,
        forks
      });
    });

    return repositories;
  } catch (error) {
    console.error(`爬取Repositories失败 (${since}): ${error.message}, URL: ${url}`);
    return [];
  }
}

// 爬取Trending Developers
async function crawlDevelopers(since = 'daily') {
  const url = since === 'daily' ? 'https://github.com/trending/developers' : `https://github.com/trending/developers?since=${since}`;
  try {
    const response = await api.get(url);
    const $ = cheerio.load(response.data);
    const developers = [];

    $('.Box-row').each((index, element) => {
      const dev = $(element);
      const avatarUrl = (dev.find('img.avatar-user').attr('src') || dev.find('img.avatar').attr('src') || '').replace(/^\/\//, 'https://');
      if (!avatarUrl) console.log(`[DEBUG] 未找到开发者${username}的头像URL`);
      const fullName = dev.find('.h3 a').text().trim() || '';
      const username = dev.find('.f4.text-normal').text().trim().replace('@', '') || '';
      const profileUrl = `https://github.com${dev.find('.h3 a').attr('href') || ''}`;
      // 修复类名拼写错误 (repo-snipit -> repo-snippet)
      const repoName = dev.find('h1.h4 a').text().trim() || '';
      const repoDescription = dev.find('.f6.color-fg-muted.mt-1').text().trim() || '';

      developers.push({
        avatarUrl,
        fullName,
        username,
        profileUrl,
        popularRepo: {
          name: repoName,
          description: repoDescription
        }
      });
    });

    return developers;
  } catch (error) {
    console.error(`爬取Developers失败 (${since}): ${error.message}, URL: ${url}`);
    return [];
  }
}

// 保存数据到JSON文件
async function saveTrendingData() {
  try {
    // 爬取日、周、月三个维度的数据
    const timePeriods = ['daily', 'weekly', 'monthly'];
    const dateStr = new Date().toISOString().split('T')[0];
    const delayBetweenRequests = () => new Promise(resolve => setTimeout(resolve, Math.random() * 5000 + 3000)); // 3-8秒随机延迟

    for (const period of timePeriods) {
      const [repositories, developers] = await Promise.all([
        crawlRepositories(period),
        crawlDevelopers(period)
      ]);

      // 检查是否爬取到有效数据
      if (repositories.length === 0 || developers.length === 0) {
        console.log(`[WARNING] ${period}数据爬取不完整，跳过文件生成`);
        continue;
      }

      const data = {
        timestamp: new Date().toISOString(),
        period,
        repositories,
        developers
      };

      const filePath = path.join(dataDir, `github-trending-${period}-${dateStr}.json`);

      try {
        // 删除同周期的旧数据文件
        const oldFiles = fs.readdirSync(dataDir)
          .filter(file => file.startsWith(`github-trending-${period}-`) && file.endsWith('.json'))
          .map(file => path.join(dataDir, file));

        oldFiles.forEach(file => {
          fs.unlinkSync(file);
          console.log(`已删除旧数据文件: ${file}`);
        });

        // 写入新文件
        fs.writeFileSync(filePath, JSON.stringify(data, null, 2));
        console.log(`Trending ${period}数据已保存: ${filePath}`);
      } catch (error) {
        console.error(`处理${period}数据文件时出错:`, error.message);
        continue;
      }

      // 不同时间段请求之间添加随机延迟
      if (period !== timePeriods[timePeriods.length - 1]) {
        console.log(`等待${(Math.random() * 5 + 3).toFixed(1)}秒后继续下一个请求...`);
        await delayBetweenRequests();
      }
    }
  } catch (error) {
    console.error('保存数据失败:', error.message);
  }
}

// 设置定时任务 - 每天凌晨1点执行
const job = schedule.scheduleJob('0 0 1 * * *', saveTrendingData);
console.log('GitHub Trending定时爬虫已启动，每天凌晨1点执行日、周、月数据爬取');

// 立即执行一次用于测试
saveTrendingData();